package com.chance.cc.crawler.development.bootstrap.zhihu.article;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.junit.Test;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.security.NoSuchAlgorithmException;

import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

/**
 * @ClassName song
 * @Description TODO
 * @Author ding
 * @Date 2021/9/1 14:22
 * @Version 1.0
 *
 **/
public class ZHArticleStart {
    private static final String domain = "zhihu";
    private static final String site = "article";
    private static Proxy proxy = new Proxy();


    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {
      // article();
        articleText();
    }
    public static void  articleText(){
        String url = "https://www.zhihu.com/topic/21695306/newest";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpConfig(HttpConfig.me(domain))
                .httpUrl(url)
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .proxy(proxy)
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);
        crawlerRequestRecord.tagsCreator().bizTags().addSiteBiz(site);

        CrawlerRecord filterRecord = new CrawlerRecord();
        filterRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        filterRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24*1,null));
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("commont_filter_tag", JSON.toJSONString(filterRecord));

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
                .consoleResultPipeline("kafka")
                .fileResultPipeline("redis","F:\\chance_log\\zhihu\\redis.log",false)
                .fileResultPipeline("kafka","F:\\chance_log\\zhihu\\kafka.log",false)
                .requestRecord(crawlerRequestRecord)
                .build("com.chance.cc.crawler.development.scripts.zhihu");
        devCrawlerController.start();
    }
    public static void  article(){
        String url = "https://www.zhihu.com/";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpConfig(HttpConfig.me(domain))
                .httpUrl(url)
                .httpHead("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36")
                .httpHead("cookie","d_c0=\"ADBiHqMywQ6PTlAq-wLKq4redBYLDaR06Hw=|1546263453\"; _xsrf=kYQKJQ39BrawjmneAv07aXGrmZhmHvaz; _ga=GA1.2.1328271267.1582701249; _zap=61c8630f-164c-4912-b2e3-11b8a48cbb83; captcha_session_v2=\"2|1:0|10:1617876364|18:captcha_session_v2|88:bXRFb1FuNmNXTHZKRWl1ZTdoYUpqcVhzRVFRcHpPdTB4WmdJWnpDbFNxeFhmaDBGUFVhMTNQUmxRdFYzamNlVg==|ec41feb8d9cea6edd33b6022246bc5d11fc6848481cb0505a07e28ffde522fa6\"; __snaker__id=1wqY9sxpGpxAdxuU; gdxidpyhxdE=juGf7TvV3b0cN5bbsawbBpMs%5CacehfjL6OK4q9r4Q7RRLjVuOq5%5CrK6cNd0fw78%2FvmDeZOnJ4mwfqm%2B6IZQmqGDJsmW%5CKd5G2NsY5Zqz4wvDDi%2Fd0JGg0SiBjDc4hviDHlQ%2FrkzBYQs0ih32m%2BCyS00yYZUZb8qh%2FX4R8xSjTmbtVvn3%3A1617877266145; _9755xjdesxxd_=32; YD00517437729195%3AWM_TID=i0Pf3X5wrfJERAFVBVM7kI4V7gwbITFX; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6eea2f463bce7bbb0b865989e8eb7c45a938e9eaaf53c9beb8fd0ea3ba2bc88b2d72af0fea7c3b92a8de7a68aeb52a1aa9a91b880b8aea9d8d4348fee0097ec41bbb2f983bb7bf3b68fb7aa4094efa1cccc4dac8c9c92f24ef79899a9e86f82b68ab9b7339294baaee240edbc9d91f14d9ce9a3bbb57d93eca299c65bb3f5bfd6d352878685a5ed4f9a8faba9db5aa3979d92c167e9ec9695f8688df1a494d86e8d8efd84fc3ab4bc9ea6d037e2a3; YD00517437729195%3AWM_NI=O9jihZcvVp5ttrg7Q8M20viKyn7b8q1jAV%2B4P1o%2BIGhTTqsF5HWHeeBjFX49RLvMc4%2FdSprGX7mZ6wQ4nxGQq%2Bq5PZEZKDnEZQeHXRaISqBm%2B4ZtulO%2FlsUfOR3cy2pdREM%3D; captcha_ticket_v2=\"2|1:0|10:1617876405|17:captcha_ticket_v2|704:eyJ2YWxpZGF0ZSI6IkNOMzFfWjFVVmU0a2YuWFlVQ3dGZzZ3Z2x5UERMOU0ueHoyblpMb3JSbEQ1NXFINkhKSHQwdTJqdWJPa3F4OW1xVnpwSkduWDVEcndWSXlIeHFSLXpNZ2dwQkdtcVlfc0FLVjZTSTRNZzFnQ1FNak1SbzJSZXZhUEhiUEJITHluXy1fNVh2eEVsZWJzY245S3o1STRWTUFvMjYuck91SjlVcENlTjZZUGx3Zmxxd1lIbU1GVXdQVExvLkFOZ2lsa20uV1VxTEp2V2dQMTV5U3luT0xObFFBR0RnR21uRVlwOWhqZFpjdkpqTU1hVk5LU0t2UGNFMVVHNVhNTERtSDFLQVhCRTBITDdaSU10Lm5BQnE2X053ODB6T2syeGVqN0VlZklHZ2FHSzFjelFDZzZjLkhQcEdBLXVVUEZPTHAtS3lDaDBmRFNrRWhSVmRhem91UUN6dHNPZWp0eng1S2FsWFBpRldNN3ZsX2NqbTFScTBCOENBWkJpWENzQkdPRlFSOUdtemhqVU5mbnBxNF9Ub0x1OHdDZ29iZFphbnNOMDlyMTBBSExISVhoamd5bS5TN1UtLXUxLjUxYUItWmZYTW56YmJ4cWlGVl94Q2pNWHZISkNMbTQyaFA4UWQ0N3hVejY3dGFuYXlYOFFxd1licmhpMW5ndHd4NXN1c0RYMyJ9|229a82129a5e888c4a313fc75fffbf2c2949fc0ca954af164dc1d503f872344c\"; z_c0=\"2|1:0|10:1617876406|4:z_c0|92:Mi4xSnpwYklBQUFBQUFBTUdJZW96TEJEaVlBQUFCZ0FsVk50aU5jWVFCV1ZuOHZybWR4a0RFb2c5eVp4dHdfV2p3MG1R|c2947f184094fd480ce7665ec5bcb710603e26c16d1892adea2553b607f9ce0d\"; tst=r; q_c1=8a20d8ca018d48a9be837a64a594eb25|1617877410000|1575450284000; __utma=51854390.1328271267.1582701249.1600809316.1617877411.9; __utmz=51854390.1617877411.9.1.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/question/47696522/answer/1526610644; __utmv=51854390.110--|2=registration_date=20201028=1^3=entry_date=20191204=1; SESSIONID=pssyzSdBpQ0wFcsaUlgKlSNnDkUuMkpODv9e9U0Pf6E; osd=V1sXAUnsTVqRM9m1fOsMSw2MZTBqmj8p6E6M0D6bGmvmdb3AHGrxNfYy3rR8sLV1jCmRt-pb4Wim-HlmEr3WlkA=; JOID=W1wXBULgSlqVONWyfO8HRwqMYTtmnT8t40KL0DqQFmzmcbbMG2r1Pvo13rB3vLJ1iCKdsOpf6mSh-H1tHrrWkks=; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1617876365,1617878301,1617934556,1617937287; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1617943893; KLBRSID=b33d76655747159914ef8c32323d16fd|1617943897|1617943078")
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null))
                .proxy(proxy)
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);
        crawlerRequestRecord.tagsCreator().bizTags().addSiteBiz(site);

        CrawlerRecord filterRecord = new CrawlerRecord();
        filterRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        filterRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null));
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("commont_filter_tag", JSON.toJSONString(filterRecord));

        CrawlerRequestRecord keyRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPageItem)
                .httpUrl("http://192.168.1.215:9599/v1/meta/zhihu/keys?site=article")  //todo 关键字传入
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.supportSource)
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                .build();


        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
                .consoleResultPipeline("kafka")
                .fileResultPipeline("redis","F:\\chance_log\\zhihu\\redis.log",false)
                .fileResultPipeline("kafka","F:\\chance_log\\zhihu\\kafka.log",false)
                .requestRecord(crawlerRequestRecord)
                .supportRecord(keyRecord)
                .build("com.chance.cc.crawler.development.scripts.zhihu");
        devCrawlerController.start();
    }
    @Test
    public void test(){
        String url = "https://www.zhihu.com/question/481246739/answer/2096150374";
        url = url.substring(5,url.length());
        System.out.println(url);
    }
    @Test
    public void tst(){
        genSign86Headers();
    }
    private void genSign86Headers() {
        //2.0_aTN0Q7e0oR2pkMt0KXxBe0uqcLFYNuYqzhNq6ALBb_xf
                  //  /api/v5.1/topics/21695306/feeds/timeline_activity?offset=20&limit=10&data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.is_normal,comment_count,voteup_count,content,relevant_info,excerpt.author.badge[?(type=best_answerer)].topics&include=data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.content,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp
        //String api = "/api/v5.1/topics/21695306/feeds/timeline_activity?offset=20&limit=10&include=data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.content,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp&data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.is_normal,comment_count,voteup_count,content,relevant_info,excerpt.author.badge[?(type=best_answerer)].topics";
        String api = "/api/v5.1/topics/21695306/feeds/timeline_activity?offset=20&limit=10&include=data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.content,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp&data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.is_normal,comment_count,voteup_count,content,relevant_info,excerpt.author.badge[?(type=best_answerer)].topics";
        String dc0 = "\"APDdLoQDqBOPThRvoqfcqQ2B3rku5G3xIfU=|1630472725\"";
        String x_zse_93 = "101_3_2.0";
        //String x_zst_81 = "3_2.0aR_sn77yn6O92wOB8hPZnQr0EMYxc4f18wNBUgpTr7tuSRFqK6P0E69y-LS9-hp1DufI-we8gGHPgJO1xuPZ0GxCTJHR7820XM20cLRGDJXfgGCBxupMuD_Ie8FL7AtqM6O1VDQyQ6nxrRPCHukMoCXBEgOsiRP0XL2ZUBXmDDV9qhnyTXFMnXcTF_ntRueThTSY-rrMeAXBWC3qiUpLyrLfiGtLuvS0pwV1Fr9CQwNO_JHCKDHBhBHmOGLPv_YYwUpMaCeYk9cupcxfpCg8S0w1EDSMAGxy1COfaCeYygoMQUF8jUCOeALmcHVYoBp9-GH0Vqom9gHXyG3fwCXf2weGF9OmQLCMlCXMuCXCoLe18uY04qfzeHCM2LFs-vX9_vc_rhOmg9XpjgNmEwN1oGxMgGNmQJH_Ygg1WUeYCgcMBGxMDGSGTuH9tBYsBgOfFg2YoDCMtUCfWGp0Y7eGBJwCzUo1QwcquqHLWrNC";
        //String x_zst_81 = "3_2.0aR_sn77yn6O92wOB8hPZnQr0EMYxc4f18wNBUgpTr7tuSRFqK6P0E69y-LS9-hp1DufI-we8gGHPgJO1xuPZ0GxCTJHR7820XM20cLRGDJXfgGCBxupMuD_Ie8FL7AtqM6O1VDQyQ6nxrRPCHukMoCXBEgOsiRP0XL2ZUBXmDDV9qhnyTXFMnXcTF_ntRueTh7gfBqcCoDoOWgw0WBOGUhr_TBtLrUHOYiVfYU3L_wF_6HFL5gOCDrLCkHCPvRFKGwtqzbpGLJeO_9O889o_2Dc0IDxYnwV_D9g99UCm1cX9eXXOzgSGBUpLJ0N8frxOUBeCCuXLnhLsIhL81GxCbgHmJUcsPggqTbOBcDoqeJO18CL0vqfz6RgKk72u9B31MXoYirUMivHXQ9F_RqOOccg_fwYYkbxmBbx0qhOBe92MYcHqBCXCO9efww3xa9XO6Gc9IJLMaUC1NrSCXCC_WJH8kqcu_utmuhLCurNC";
        String x_zst_81 = "3_2.0aR_sn77yn6O92wOB8hPZnQr0EMYxc4f18wNBUgpTr7tuSRFqK6P0E69y-LS9-hp1DufI-we8gGHPgJO1xuPZ0GxCTJHR7820XM20cLRGDJXfgGCBxupMuD_Ie8FL7AtqM6O1VDQyQ6nxrRPCHukMoCXBEgOsiRP0XL2ZUBXmDDV9qhnyTXFMnXcTF_ntRueThL2fkirfpwNYjg30hCCLIwXqucLOCwHGGqeYfrSMXgoXcuVB2hLynhXy6_OPv0O9nrxCAbCyegYKYBL8xCxOBh98w9XYBDo_hUXqrbo8OcxLJJS9FcH0JvHmuu3moUc_RupG2RNCSXeXCqO_lBwLhwOytgt1kBY9vBe8nwxCsJx1fGxm4qfz9vV9aq2xecnOy9g8PJwmLuc8JTOflqOB8go8wcL8YBgLvD3M6eofyBtxHUS1s_HKaqL13u38IgC1dvgC9JVLLhoMZcn0ywL1JirChCHBhqgK10XOwJUC";
       // String needEncodeStr = StringUtils.joinWith("+",x_zse_93,api,dc0,x_zst_81);
     String needEncodeStr = "101_3_2.0+/api/v4/search/preset_words+\"APDdLoQDqBOPThRvoqfcqQ2B3rku5G3xIfU=|1630472725\"+3_2.0aR_sn77yn6O92wOB8hPZnQr0EMYxc4f18wNBUgpTeLtu60FqK6P0E49y-LS9-hp1DufI-we8gGHPgJO1xuPZ0GxCTJHR7820XM20cLRGDJXfgGCBxupMuD_Ie8FL7AtqM6O1VDQyQ6nxrRPCHukMoCXBEgOsiRP0XL2ZUBXmDDV9qhnyTXFMnXcTF_ntRueThTSLp9X1oBe9QgXqmgoM0bgGrRC_Lgg8YHe1qcp1eQHV28H0hGF9SQVKe0CPvMo8ErL9ZgCZQLwxCwYBtGCfbroGCbXKDGVL-gYfUcw9SM29LwHB-he9cJgqo0pLq9pKPC3MUG_9aU2KkUY_fqLOxvXqicNK-COYRBxfJ49M6Cc1fwH95qfz2gX9ZGO8eCt0-htBAvg1Vce_8uYfHhg1CJ39SHL8oJLMNbxCwDo9xwSVrGFBBgpYehX1EG2OsCHmBqgG3JxKXUtmNvw18Rw1zh91N9eVoLpG5wLLWrNC";
       String md5Code = null;
        try {
           md5Code = md5(needEncodeStr.getBytes());

        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
        String signCode = downloadSign(md5Code);
        System.out.println("x-zse-96 *****   2.0_" + signCode);
        //heads.put("x-zse-96","2.0_" + signCode);
    }
    private String downloadSign(String md5Code) {
        String sign86 = "";
        //String url = "http://192.168.1.210:8899/encrypt/zhihu" + "?code=" + md5Code;
        String url = "http://127.0.0.1:8899/encrypt/zhihu"+"?code="+md5Code;
        HttpConfig httpConfig = HttpConfig.me("sign");
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(url);
        try {
            HttpClientDownloader downloader = new HttpClientDownloader();
            downloader.setClientGenerator(new HttpClientGenerator());
            downloader.setProxyProvider(new SiteConfigProxyProvider());
            HttpPage httpPage = downloader.download(httpRequest, httpConfig);
            JSONObject jsonObject = JSONObject.parseObject(httpPage.getRawText());
            sign86 = jsonObject.getString("sign");
        } catch (Exception e) {
        }
        return sign86;
    }

    @Test
    public void tt() throws UnsupportedEncodingException {
        String a = "/api/v5.1/topics/21695306/feeds/timeline_activity?offset=10&limit=10&include=";
        String u = "data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.content,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp;data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.is_normal,comment_count,voteup_count,content,relevant_info,excerpt.author.badge[?(type=best_answerer)].topics;data[?(target.type=topic_sticky_module)].target.data[?(target.type=article)].target.content,voteup_count,comment_count,voting,author.badge[?(type=best_answerer)].topics;data[?(target.type=topic_sticky_module)].target.data[?(target.type=people)].target.answer_count,articles_count,gender,follower_count,is_followed,is_following,badge[?(type=best_answerer)].topics;data[?(target.type=answer)].target.annotation_detail,content,hermes_label,is_labeled,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp,answer_type;data[?(target.type=answer)].target.author.badge[?(type=best_answerer)].topics;data[?(target.type=answer)].target.paid_info;data[?(target.type=article)].target.annotation_detail,content,hermes_label,is_labeled,author.badge[?(type=best_answerer)].topics;data[?(target.type=question)].target.annotation_detail,comment_count;";
        String e = URLEncoder.encode(u,"UTF-8");
        System.out.println(a+e);
    }
}
